{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Concatenate TCGA MAF Files\n",
    "This notebook loads all the MAF files downloaded from the TCGA data portal and joins them. Duplicates are removed prior to analyzing the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import os\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Joining all MAF Files\n",
    "First, let's try to read all the MAF files and concatenate them. After that, I should be left with one big MAF that I can write to file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (87,98,118) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] Got 64804 mutations and 120 common columns\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (87,88,98,118) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lost 18 columns reading ../../data/pancancer/TCGA/mutation/download_new/c06465a3-50e7-46f7-b2dd-7bd654ca206b/TCGA.STAD.mutect.c06465a3-50e7-46f7-b2dd-7bd654ca206b.DR-10.0.somatic.maf\n",
      "Lost columns: ['Match_Norm_Seq_Allele1', 'Match_Norm_Seq_Allele2', 'Match_Norm_Validation_Allele1', 'Match_Norm_Validation_Allele2', 'Verification_Status', 'Validation_Status', 'Sequencing_Phase', 'Sequence_Source', 'Score', 'BAM_File', 'n_ref_count', 'n_alt_count', 'ASN_MAF', 'MOTIF_NAME', 'MOTIF_POS', 'HIGH_INF_POS', 'MOTIF_SCORE_CHANGE', 'GENE_PHENO']\n",
      "New Maf cols: Index(['Hugo_Symbol', 'Entrez_Gene_Id', 'Center', 'NCBI_Build', 'Chromosome',\n",
      "       'Start_Position', 'End_Position', 'Strand', 'Variant_Classification',\n",
      "       'Variant_Type',\n",
      "       ...\n",
      "       'FILTER', 'CONTEXT', 'src_vcf_id', 'tumor_bam_uuid', 'normal_bam_uuid',\n",
      "       'case_id', 'GDC_FILTER', 'COSMIC', 'MC3_Overlap',\n",
      "       'GDC_Validation_Status'],\n",
      "      dtype='object', length=120)\n",
      "Old MAF cols: Index(['Hugo_Symbol', 'Entrez_Gene_Id', 'Center', 'NCBI_Build', 'Chromosome',\n",
      "       'Start_Position', 'End_Position', 'Strand', 'Variant_Classification',\n",
      "       'Variant_Type',\n",
      "       ...\n",
      "       'FILTER', 'CONTEXT', 'src_vcf_id', 'tumor_bam_uuid', 'normal_bam_uuid',\n",
      "       'case_id', 'GDC_FILTER', 'COSMIC', 'MC3_Overlap',\n",
      "       'GDC_Validation_Status'],\n",
      "      dtype='object', length=120)\n",
      "Join MAF cols: Index(['Hugo_Symbol', 'Entrez_Gene_Id', 'Center', 'NCBI_Build', 'Chromosome',\n",
      "       'Start_Position', 'End_Position', 'Strand', 'Variant_Classification',\n",
      "       'Variant_Type',\n",
      "       ...\n",
      "       'FILTER', 'CONTEXT', 'src_vcf_id', 'tumor_bam_uuid', 'normal_bam_uuid',\n",
      "       'case_id', 'GDC_FILTER', 'COSMIC', 'MC3_Overlap',\n",
      "       'GDC_Validation_Status'],\n",
      "      dtype='object', length=102)\n",
      "[2] Got 277948 mutations and 102 common columns\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (87,88,118) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lost 18 columns reading ../../data/pancancer/TCGA/mutation/download_new/0458c57f-316c-4a7c-9294-ccd11c97c2f9/TCGA.LUAD.mutect.0458c57f-316c-4a7c-9294-ccd11c97c2f9.DR-10.0.somatic.maf\n",
      "Lost columns: ['Match_Norm_Seq_Allele1', 'Match_Norm_Seq_Allele2', 'Match_Norm_Validation_Allele1', 'Match_Norm_Validation_Allele2', 'Verification_Status', 'Validation_Status', 'Sequencing_Phase', 'Sequence_Source', 'Score', 'BAM_File', 'n_ref_count', 'n_alt_count', 'ASN_MAF', 'MOTIF_NAME', 'MOTIF_POS', 'HIGH_INF_POS', 'MOTIF_SCORE_CHANGE', 'GENE_PHENO']\n",
      "New Maf cols: Index(['Hugo_Symbol', 'Entrez_Gene_Id', 'Center', 'NCBI_Build', 'Chromosome',\n",
      "       'Start_Position', 'End_Position', 'Strand', 'Variant_Classification',\n",
      "       'Variant_Type',\n",
      "       ...\n",
      "       'FILTER', 'CONTEXT', 'src_vcf_id', 'tumor_bam_uuid', 'normal_bam_uuid',\n",
      "       'case_id', 'GDC_FILTER', 'COSMIC', 'MC3_Overlap',\n",
      "       'GDC_Validation_Status'],\n",
      "      dtype='object', length=120)\n",
      "Old MAF cols: Index(['Hugo_Symbol', 'Entrez_Gene_Id', 'Center', 'NCBI_Build', 'Chromosome',\n",
      "       'Start_Position', 'End_Position', 'Strand', 'Variant_Classification',\n",
      "       'Variant_Type',\n",
      "       ...\n",
      "       'FILTER', 'CONTEXT', 'src_vcf_id', 'tumor_bam_uuid', 'normal_bam_uuid',\n",
      "       'case_id', 'GDC_FILTER', 'COSMIC', 'MC3_Overlap',\n",
      "       'GDC_Validation_Status'],\n",
      "      dtype='object', length=102)\n",
      "Join MAF cols: Index(['Hugo_Symbol', 'Entrez_Gene_Id', 'Center', 'NCBI_Build', 'Chromosome',\n",
      "       'Start_Position', 'End_Position', 'Strand', 'Variant_Classification',\n",
      "       'Variant_Type',\n",
      "       ...\n",
      "       'FILTER', 'CONTEXT', 'src_vcf_id', 'tumor_bam_uuid', 'normal_bam_uuid',\n",
      "       'case_id', 'GDC_FILTER', 'COSMIC', 'MC3_Overlap',\n",
      "       'GDC_Validation_Status'],\n",
      "      dtype='object', length=102)\n",
      "[3] Got 486128 mutations and 102 common columns\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (19,20,28,87,88) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4] Got 589533 mutations and 102 common columns\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (87) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5] Got 634846 mutations and 102 common columns\n",
      "Exception occured while reading ../../data/pancancer/TCGA/mutation/download_new/d3fa70be-520a-420e-bb6d-651aeee5cb50/TCGA.UCEC.mutect.d3fa70be-520a-420e-bb6d-651aeee5cb50.DR-10.0.somatic.maf... Skipping!\n",
      "[5] Got 634846 mutations and 102 common columns\n",
      "[6] Got 755834 mutations and 102 common columns\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (88,118) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[7] Got 810072 mutations and 102 common columns\n",
      "[8] Got 833837 mutations and 102 common columns\n",
      "[9] Got 1098623 mutations and 102 common columns\n",
      "[10] Got 1279739 mutations and 102 common columns\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (87,118) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[11] Got 1382048 mutations and 102 common columns\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (88,98) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[12] Got 1392947 mutations and 102 common columns\n",
      "[13] Got 1527460 mutations and 102 common columns\n",
      "[14] Got 1554153 mutations and 102 common columns\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count = 0\n",
    "maf_all = None\n",
    "col_names = None\n",
    "root_dir = '../../data/pancancer/TCGA/mutation/download_new/'\n",
    "for subdir, d, files in os.walk(root_dir):\n",
    "    for fname in files:\n",
    "        if fname.endswith('.maf'):\n",
    "            p = os.path.join(subdir, fname)\n",
    "            if maf_all is None:\n",
    "                maf_all = pd.read_csv(p, sep='\\t', comment='#', header=0)\n",
    "                col_names = maf_all.columns\n",
    "            else:\n",
    "                print (\"[{}] Got {} mutations and {} common columns\".format(count, maf_all.shape[0], maf_all.shape[1]))\n",
    "                try:\n",
    "                    maf_new = pd.read_csv(p, sep='\\t', comment='#', header=0)\n",
    "                except:\n",
    "                    print (\"Exception occured while reading {}... Skipping!\".format(p))\n",
    "                    continue\n",
    "                if {'Hugo_Symbol', 'Variant_Classification', 'Tumor_Sample_Barcode'}.issubset(maf_new.columns):\n",
    "                    maf_join = pd.concat([maf_all, maf_new], ignore_index=True, join='inner')\n",
    "                    maf_join.dropna(axis=1, inplace=True, how='all') # remove stupid all-na columns\n",
    "                    if len(maf_join.columns) < len(col_names):\n",
    "                        print (\"Lost {} columns reading {}\".format(len(col_names)-maf_join.shape[1], p))\n",
    "                        print (\"Lost columns: {}\".format([c for c in col_names if not c in maf_join.columns]))\n",
    "                        print (\"New Maf cols: {}\".format(maf_new.columns))\n",
    "                        print (\"Old MAF cols: {}\".format(maf_all.columns))\n",
    "                        print (\"Join MAF cols: {}\".format(maf_join.columns))\n",
    "                        col_names = maf_all.columns\n",
    "                    maf_all = maf_join\n",
    "                else:\n",
    "                    print (\"File contains not all columns needed or is empty {}\".format(p))\n",
    "            count += 1\n",
    "count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Hugo_Symbol</th>\n",
       "      <th>Entrez_Gene_Id</th>\n",
       "      <th>Center</th>\n",
       "      <th>NCBI_Build</th>\n",
       "      <th>Chromosome</th>\n",
       "      <th>Start_Position</th>\n",
       "      <th>End_Position</th>\n",
       "      <th>Strand</th>\n",
       "      <th>Variant_Classification</th>\n",
       "      <th>Variant_Type</th>\n",
       "      <th>...</th>\n",
       "      <th>FILTER</th>\n",
       "      <th>CONTEXT</th>\n",
       "      <th>src_vcf_id</th>\n",
       "      <th>tumor_bam_uuid</th>\n",
       "      <th>normal_bam_uuid</th>\n",
       "      <th>case_id</th>\n",
       "      <th>GDC_FILTER</th>\n",
       "      <th>COSMIC</th>\n",
       "      <th>MC3_Overlap</th>\n",
       "      <th>GDC_Validation_Status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C1orf94</td>\n",
       "      <td>84970</td>\n",
       "      <td>BCM</td>\n",
       "      <td>GRCh38</td>\n",
       "      <td>chr1</td>\n",
       "      <td>34218878</td>\n",
       "      <td>34218878</td>\n",
       "      <td>+</td>\n",
       "      <td>3'UTR</td>\n",
       "      <td>SNP</td>\n",
       "      <td>...</td>\n",
       "      <td>PASS</td>\n",
       "      <td>GGTCACAGACA</td>\n",
       "      <td>a6945d7d-472c-4da7-a122-de6d014564e9</td>\n",
       "      <td>12f0b883-900e-41b4-bbdf-a66b96e825ca</td>\n",
       "      <td>8d36e3fc-bd2c-441d-a4e1-ed7c94d2cb3e</td>\n",
       "      <td>e6827400-0d95-46b0-8874-6ce9e9d5011b</td>\n",
       "      <td>wga_pair</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>Unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MACF1</td>\n",
       "      <td>23499</td>\n",
       "      <td>BCM</td>\n",
       "      <td>GRCh38</td>\n",
       "      <td>chr1</td>\n",
       "      <td>39357837</td>\n",
       "      <td>39357837</td>\n",
       "      <td>+</td>\n",
       "      <td>Missense_Mutation</td>\n",
       "      <td>SNP</td>\n",
       "      <td>...</td>\n",
       "      <td>PASS</td>\n",
       "      <td>GAAACTTAGTA</td>\n",
       "      <td>a6945d7d-472c-4da7-a122-de6d014564e9</td>\n",
       "      <td>12f0b883-900e-41b4-bbdf-a66b96e825ca</td>\n",
       "      <td>8d36e3fc-bd2c-441d-a4e1-ed7c94d2cb3e</td>\n",
       "      <td>e6827400-0d95-46b0-8874-6ce9e9d5011b</td>\n",
       "      <td>wga_pair</td>\n",
       "      <td>COSM288283;COSM288284</td>\n",
       "      <td>True</td>\n",
       "      <td>Unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GBP5</td>\n",
       "      <td>115362</td>\n",
       "      <td>BCM</td>\n",
       "      <td>GRCh38</td>\n",
       "      <td>chr1</td>\n",
       "      <td>89260749</td>\n",
       "      <td>89260749</td>\n",
       "      <td>+</td>\n",
       "      <td>Silent</td>\n",
       "      <td>SNP</td>\n",
       "      <td>...</td>\n",
       "      <td>PASS</td>\n",
       "      <td>TGGGCGTGCTG</td>\n",
       "      <td>a6945d7d-472c-4da7-a122-de6d014564e9</td>\n",
       "      <td>12f0b883-900e-41b4-bbdf-a66b96e825ca</td>\n",
       "      <td>8d36e3fc-bd2c-441d-a4e1-ed7c94d2cb3e</td>\n",
       "      <td>e6827400-0d95-46b0-8874-6ce9e9d5011b</td>\n",
       "      <td>wga_pair</td>\n",
       "      <td>COSM288277</td>\n",
       "      <td>True</td>\n",
       "      <td>Unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ZNF326</td>\n",
       "      <td>284695</td>\n",
       "      <td>BCM</td>\n",
       "      <td>GRCh38</td>\n",
       "      <td>chr1</td>\n",
       "      <td>90007501</td>\n",
       "      <td>90007501</td>\n",
       "      <td>+</td>\n",
       "      <td>Silent</td>\n",
       "      <td>SNP</td>\n",
       "      <td>...</td>\n",
       "      <td>PASS</td>\n",
       "      <td>TCTTTCGGAGG</td>\n",
       "      <td>a6945d7d-472c-4da7-a122-de6d014564e9</td>\n",
       "      <td>12f0b883-900e-41b4-bbdf-a66b96e825ca</td>\n",
       "      <td>8d36e3fc-bd2c-441d-a4e1-ed7c94d2cb3e</td>\n",
       "      <td>e6827400-0d95-46b0-8874-6ce9e9d5011b</td>\n",
       "      <td>wga_pair</td>\n",
       "      <td>COSM288313</td>\n",
       "      <td>True</td>\n",
       "      <td>Unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>BRDT</td>\n",
       "      <td>676</td>\n",
       "      <td>BCM</td>\n",
       "      <td>GRCh38</td>\n",
       "      <td>chr1</td>\n",
       "      <td>91977358</td>\n",
       "      <td>91977358</td>\n",
       "      <td>+</td>\n",
       "      <td>Missense_Mutation</td>\n",
       "      <td>SNP</td>\n",
       "      <td>...</td>\n",
       "      <td>PASS</td>\n",
       "      <td>ATGACGTTGTC</td>\n",
       "      <td>a6945d7d-472c-4da7-a122-de6d014564e9</td>\n",
       "      <td>12f0b883-900e-41b4-bbdf-a66b96e825ca</td>\n",
       "      <td>8d36e3fc-bd2c-441d-a4e1-ed7c94d2cb3e</td>\n",
       "      <td>e6827400-0d95-46b0-8874-6ce9e9d5011b</td>\n",
       "      <td>wga_pair</td>\n",
       "      <td>COSM171492</td>\n",
       "      <td>True</td>\n",
       "      <td>Unknown</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 102 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  Hugo_Symbol  Entrez_Gene_Id Center NCBI_Build Chromosome  Start_Position  \\\n",
       "0     C1orf94           84970    BCM     GRCh38       chr1        34218878   \n",
       "1       MACF1           23499    BCM     GRCh38       chr1        39357837   \n",
       "2        GBP5          115362    BCM     GRCh38       chr1        89260749   \n",
       "3      ZNF326          284695    BCM     GRCh38       chr1        90007501   \n",
       "4        BRDT             676    BCM     GRCh38       chr1        91977358   \n",
       "\n",
       "   End_Position Strand Variant_Classification Variant_Type  ... FILTER  \\\n",
       "0      34218878      +                  3'UTR          SNP  ...   PASS   \n",
       "1      39357837      +      Missense_Mutation          SNP  ...   PASS   \n",
       "2      89260749      +                 Silent          SNP  ...   PASS   \n",
       "3      90007501      +                 Silent          SNP  ...   PASS   \n",
       "4      91977358      +      Missense_Mutation          SNP  ...   PASS   \n",
       "\n",
       "       CONTEXT                            src_vcf_id  \\\n",
       "0  GGTCACAGACA  a6945d7d-472c-4da7-a122-de6d014564e9   \n",
       "1  GAAACTTAGTA  a6945d7d-472c-4da7-a122-de6d014564e9   \n",
       "2  TGGGCGTGCTG  a6945d7d-472c-4da7-a122-de6d014564e9   \n",
       "3  TCTTTCGGAGG  a6945d7d-472c-4da7-a122-de6d014564e9   \n",
       "4  ATGACGTTGTC  a6945d7d-472c-4da7-a122-de6d014564e9   \n",
       "\n",
       "                         tumor_bam_uuid                       normal_bam_uuid  \\\n",
       "0  12f0b883-900e-41b4-bbdf-a66b96e825ca  8d36e3fc-bd2c-441d-a4e1-ed7c94d2cb3e   \n",
       "1  12f0b883-900e-41b4-bbdf-a66b96e825ca  8d36e3fc-bd2c-441d-a4e1-ed7c94d2cb3e   \n",
       "2  12f0b883-900e-41b4-bbdf-a66b96e825ca  8d36e3fc-bd2c-441d-a4e1-ed7c94d2cb3e   \n",
       "3  12f0b883-900e-41b4-bbdf-a66b96e825ca  8d36e3fc-bd2c-441d-a4e1-ed7c94d2cb3e   \n",
       "4  12f0b883-900e-41b4-bbdf-a66b96e825ca  8d36e3fc-bd2c-441d-a4e1-ed7c94d2cb3e   \n",
       "\n",
       "                                case_id GDC_FILTER                 COSMIC  \\\n",
       "0  e6827400-0d95-46b0-8874-6ce9e9d5011b   wga_pair                    NaN   \n",
       "1  e6827400-0d95-46b0-8874-6ce9e9d5011b   wga_pair  COSM288283;COSM288284   \n",
       "2  e6827400-0d95-46b0-8874-6ce9e9d5011b   wga_pair             COSM288277   \n",
       "3  e6827400-0d95-46b0-8874-6ce9e9d5011b   wga_pair             COSM288313   \n",
       "4  e6827400-0d95-46b0-8874-6ce9e9d5011b   wga_pair             COSM171492   \n",
       "\n",
       "  MC3_Overlap GDC_Validation_Status  \n",
       "0        True               Unknown  \n",
       "1        True               Unknown  \n",
       "2        True               Unknown  \n",
       "3        True               Unknown  \n",
       "4        True               Unknown  \n",
       "\n",
       "[5 rows x 102 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maf_all.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/ipykernel_launcher.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n",
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/pandas/core/indexing.py:966: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.obj[item] = s\n"
     ]
    }
   ],
   "source": [
    "maf_shortened = maf_all[['Hugo_Symbol', 'Tumor_Sample_Barcode', 'Chromosome', 'Start_Position',\n",
    "                         'End_Position', 'Variant_Classification', 'Reference_Allele',\n",
    "                         'Tumor_Seq_Allele2']]\n",
    "maf_shortened.columns = ['Gene', 'Tumor_Sample', 'Chromosome', 'Start_Position', 'End_Position',\n",
    "                         'Variant_Classification', 'Reference_Allele', 'Tumor_Allele']\n",
    "maf_shortened['Gene'] = maf_shortened.Gene.str.strip()\n",
    "maf_shortened.set_index('Gene', inplace=True)\n",
    "trim_barcode_to_patient = lambda x: '-'.join(str(x).split('-')[:3]) # TCGA barcode until patient\n",
    "maf_shortened.loc[:, 'Tumor_Sample'] = maf_shortened.Tumor_Sample.map(trim_barcode_to_patient)\n",
    "#maf_shortened.to_csv('../../data/pancancer/TCGA/mutation/pancan_maf/pancancer_16_onlyrequiredcols.maf', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "maf_all.Tumor_Sample_Barcode = maf_all.Tumor_Sample_Barcode.map(trim_barcode_to_patient)\n",
    "maf_all.to_csv('../../data/pancancer/TCGA/mutation/pancan_maf/pancancer_16_samecols.maf', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (1,5,6,17,18,19,20,21,22,23,24,26,27,28,29,30,39,40,41,42,43,44,58,59,60,77,78,79,80,81,82,83,84,85,87,88,89,90,91,92,94,96,97,98,99,100,101,102,103,104,105,106,107,108,109,118) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "21766"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "verification = pd.read_csv('../../../2020plus-1.2.3/data/pancancer_16_hg19.maf', sep='\\t', comment='#', header=0)\n",
    "verification.Gene.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/ipykernel_launcher.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n",
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/pandas/core/indexing.py:1048: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.obj[item_labels[indexer[info_axis]]] = value\n"
     ]
    }
   ],
   "source": [
    "maf_shortened = verification[['Hugo_Symbol', 'Tumor_Sample_Barcode', 'Chromosome', 'Start_Position',\n",
    "                         'End_Position', 'Variant_Classification', 'Reference_Allele',\n",
    "                         'Tumor_Seq_Allele2']]\n",
    "maf_shortened.columns = ['Gene', 'Tumor_Sample', 'Chromosome', 'Start_Position', 'End_Position',\n",
    "                         'Variant_Classification', 'Reference_Allele', 'Tumor_Allele']\n",
    "maf_shortened['Gene'] = maf_shortened.Gene.str.strip()\n",
    "maf_shortened.set_index('Gene', inplace=True)\n",
    "trim_barcode_to_patient = lambda x: '-'.join(str(x).split('-')[:3]) # TCGA barcode until patient\n",
    "maf_shortened.loc[:, 'Tumor_Sample'] = maf_shortened.Tumor_Sample.map(trim_barcode_to_patient)\n",
    "#maf_shortened.to_csv('../../data/pancancer/TCGA/mutation/pancan_maf/STAD_onlyrequiredcols.maf', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "maf_shortened.to_csv('/project/gcn/2020plus-1.2.3/data/pancancer_16_short_hg19.maf', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Gene\n",
       "C1orf94     34684479\n",
       "MACF1       39823509\n",
       "GBP5        89726432\n",
       "ZNF326      90473060\n",
       "BRDT        92442915\n",
       "             ...    \n",
       "GLRA2       14599376\n",
       "PHKA2       18912456\n",
       "CYBB        37641374\n",
       "AWAT1       69458062\n",
       "AFF2       147733566\n",
       "Name: Start_Position, Length: 2469673, dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maf_shortened = maf_shortened.drop('Hugo_Symbol')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TP53       2883\n",
       "KMT2D      1343\n",
       "APC        1197\n",
       "KMT2C      1110\n",
       "ARID1A      981\n",
       "           ... \n",
       "SMARCB1     105\n",
       "GATA1        89\n",
       "NPM1         68\n",
       "CEBPA        28\n",
       "SOCS1        24\n",
       "Name: Gene, Length: 71, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oncogenes = pd.read_csv('../../../2020plus-1.2.3/data/gene_lists/oncogenes.txt', header=None)\n",
    "tsgs = pd.read_csv('../../../2020plus-1.2.3/data/gene_lists/tsgs.txt', header=None)\n",
    "oncogenes.columns = ['Name']\n",
    "tsgs.columns = ['Name']\n",
    "verification[verification.Gene.isin(tsgs.Name)].Gene.value_counts()#, verification[verification.Gene.isin(oncogenes.Name)].Gene.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collected MAF file with mutations from 6909 patients\n"
     ]
    }
   ],
   "source": [
    "print (\"Collected MAF file with mutations from {} patients\".format(maf_shortened.Tumor_Sample.nunique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Missense_Mutation         1252240\n",
       "Silent                     461766\n",
       "3'UTR                      218245\n",
       "Intron                     113019\n",
       "Nonsense_Mutation          107922\n",
       "Frame_Shift_Del             77033\n",
       "5'UTR                       52744\n",
       "RNA                         44178\n",
       "Frame_Shift_Ins             35047\n",
       "Splice_Site                 31509\n",
       "Splice_Region               23981\n",
       "3'Flank                     22011\n",
       "5'Flank                     15535\n",
       "In_Frame_Ins                 5861\n",
       "In_Frame_Del                 5349\n",
       "Translation_Start_Site       1605\n",
       "Nonstop_Mutation             1441\n",
       "IGR                           330\n",
       "Name: Variant_Classification, dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maf_shortened.Variant_Classification.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "613333"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "variant_snv = ['Missense_Mutation', 'Silent', 'Nonsense_Mutation',\n",
    "               'Splice_Site', 'Nonstop_Mutation', 'Translation_Start_Site']\n",
    "maf_shortened.shape[0] - maf_shortened[maf_shortened.Variant_Classification.isin(variant_snv)].shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Do Some Preprocessing\n",
    "After I now have one big DataFrame with all the MAF files concatenated, I want to clean it a little.\n",
    "\n",
    "I will do the following:\n",
    "* Remove all mutations without `Hugo_Symbol`. Mutations that are not in genes can't help me to compute mutation frequencies.\n",
    "* Remove duplicated rows. If a mutation is an exact duplicate of another, remove it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "maf_no_nas = maf_all.loc[~maf_all.Hugo_Symbol.isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Joined MAF contained 1859965 duplicates. Left with 1864774 out of 3724739 mutations\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/pkg/python-3.6.0-0/lib/python3.6/site-packages/pandas/util/decorators.py:91: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  return func(*args, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "no_mut_dup = maf_no_nas.shape[0]\n",
    "maf_no_nas.drop_duplicates(subset=['Hugo_Symbol', 'Variant_Classification', 'Tumor_Sample_Barcode'], inplace=True)\n",
    "print (\"Joined MAF contained {} duplicates. Left with {} out of {} mutations\".format(no_mut_dup-maf_no_nas.shape[0],\n",
    "                                                                                  maf_no_nas.shape[0], no_mut_dup))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (87,88,98,118) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Hugo_Symbol</th>\n",
       "      <th>Entrez_Gene_Id</th>\n",
       "      <th>Center</th>\n",
       "      <th>NCBI_Build</th>\n",
       "      <th>Chromosome</th>\n",
       "      <th>Start_Position</th>\n",
       "      <th>End_Position</th>\n",
       "      <th>Strand</th>\n",
       "      <th>Variant_Classification</th>\n",
       "      <th>Variant_Type</th>\n",
       "      <th>...</th>\n",
       "      <th>FILTER</th>\n",
       "      <th>CONTEXT</th>\n",
       "      <th>src_vcf_id</th>\n",
       "      <th>tumor_bam_uuid</th>\n",
       "      <th>normal_bam_uuid</th>\n",
       "      <th>case_id</th>\n",
       "      <th>GDC_FILTER</th>\n",
       "      <th>COSMIC</th>\n",
       "      <th>MC3_Overlap</th>\n",
       "      <th>GDC_Validation_Status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C1orf86</td>\n",
       "      <td>199990</td>\n",
       "      <td>BI</td>\n",
       "      <td>GRCh38</td>\n",
       "      <td>chr1</td>\n",
       "      <td>2193894</td>\n",
       "      <td>2193894</td>\n",
       "      <td>+</td>\n",
       "      <td>Frame_Shift_Del</td>\n",
       "      <td>DEL</td>\n",
       "      <td>...</td>\n",
       "      <td>PASS</td>\n",
       "      <td>GGCTCCGGGCCG</td>\n",
       "      <td>b06a171d-e6f1-4f81-b9e3-e5e4522ccf5d</td>\n",
       "      <td>003d6bd5-3ae8-4838-8cfc-0355ee16122d</td>\n",
       "      <td>b4ede00d-9733-4eb2-9335-c7bcd7976d2e</td>\n",
       "      <td>ec02b33c-8094-45e8-b679-b14dbe75d66f</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>Unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ERRFI1</td>\n",
       "      <td>54206</td>\n",
       "      <td>BI</td>\n",
       "      <td>GRCh38</td>\n",
       "      <td>chr1</td>\n",
       "      <td>8013618</td>\n",
       "      <td>8013618</td>\n",
       "      <td>+</td>\n",
       "      <td>Silent</td>\n",
       "      <td>SNP</td>\n",
       "      <td>...</td>\n",
       "      <td>panel_of_normals</td>\n",
       "      <td>TTACTCGGTGA</td>\n",
       "      <td>b06a171d-e6f1-4f81-b9e3-e5e4522ccf5d</td>\n",
       "      <td>003d6bd5-3ae8-4838-8cfc-0355ee16122d</td>\n",
       "      <td>b4ede00d-9733-4eb2-9335-c7bcd7976d2e</td>\n",
       "      <td>ec02b33c-8094-45e8-b679-b14dbe75d66f</td>\n",
       "      <td>gdc_pon</td>\n",
       "      <td>COSM3492653</td>\n",
       "      <td>True</td>\n",
       "      <td>Unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CLCN6</td>\n",
       "      <td>1185</td>\n",
       "      <td>BI</td>\n",
       "      <td>GRCh38</td>\n",
       "      <td>chr1</td>\n",
       "      <td>11833961</td>\n",
       "      <td>11833961</td>\n",
       "      <td>+</td>\n",
       "      <td>Missense_Mutation</td>\n",
       "      <td>SNP</td>\n",
       "      <td>...</td>\n",
       "      <td>PASS</td>\n",
       "      <td>TCCAAGTGGCC</td>\n",
       "      <td>b06a171d-e6f1-4f81-b9e3-e5e4522ccf5d</td>\n",
       "      <td>003d6bd5-3ae8-4838-8cfc-0355ee16122d</td>\n",
       "      <td>b4ede00d-9733-4eb2-9335-c7bcd7976d2e</td>\n",
       "      <td>ec02b33c-8094-45e8-b679-b14dbe75d66f</td>\n",
       "      <td>NaN</td>\n",
       "      <td>COSM4021197</td>\n",
       "      <td>True</td>\n",
       "      <td>Unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>PRAMEF1</td>\n",
       "      <td>65121</td>\n",
       "      <td>BI</td>\n",
       "      <td>GRCh38</td>\n",
       "      <td>chr1</td>\n",
       "      <td>12795592</td>\n",
       "      <td>12795592</td>\n",
       "      <td>+</td>\n",
       "      <td>Missense_Mutation</td>\n",
       "      <td>SNP</td>\n",
       "      <td>...</td>\n",
       "      <td>panel_of_normals</td>\n",
       "      <td>CCCTCGGAGCT</td>\n",
       "      <td>b06a171d-e6f1-4f81-b9e3-e5e4522ccf5d</td>\n",
       "      <td>003d6bd5-3ae8-4838-8cfc-0355ee16122d</td>\n",
       "      <td>b4ede00d-9733-4eb2-9335-c7bcd7976d2e</td>\n",
       "      <td>ec02b33c-8094-45e8-b679-b14dbe75d66f</td>\n",
       "      <td>gdc_pon</td>\n",
       "      <td>COSM4021467;COSM4021468;COSM4021469</td>\n",
       "      <td>True</td>\n",
       "      <td>Unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PRAMEF18</td>\n",
       "      <td>391003</td>\n",
       "      <td>BI</td>\n",
       "      <td>GRCh38</td>\n",
       "      <td>chr1</td>\n",
       "      <td>13225284</td>\n",
       "      <td>13225284</td>\n",
       "      <td>+</td>\n",
       "      <td>Missense_Mutation</td>\n",
       "      <td>SNP</td>\n",
       "      <td>...</td>\n",
       "      <td>PASS</td>\n",
       "      <td>TCAAGGGCTGC</td>\n",
       "      <td>b06a171d-e6f1-4f81-b9e3-e5e4522ccf5d</td>\n",
       "      <td>003d6bd5-3ae8-4838-8cfc-0355ee16122d</td>\n",
       "      <td>b4ede00d-9733-4eb2-9335-c7bcd7976d2e</td>\n",
       "      <td>ec02b33c-8094-45e8-b679-b14dbe75d66f</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>Unknown</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 120 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  Hugo_Symbol  Entrez_Gene_Id Center NCBI_Build Chromosome  Start_Position  \\\n",
       "0     C1orf86          199990     BI     GRCh38       chr1         2193894   \n",
       "1      ERRFI1           54206     BI     GRCh38       chr1         8013618   \n",
       "2       CLCN6            1185     BI     GRCh38       chr1        11833961   \n",
       "3     PRAMEF1           65121     BI     GRCh38       chr1        12795592   \n",
       "4    PRAMEF18          391003     BI     GRCh38       chr1        13225284   \n",
       "\n",
       "   End_Position Strand Variant_Classification Variant_Type  ...  \\\n",
       "0       2193894      +        Frame_Shift_Del          DEL  ...   \n",
       "1       8013618      +                 Silent          SNP  ...   \n",
       "2      11833961      +      Missense_Mutation          SNP  ...   \n",
       "3      12795592      +      Missense_Mutation          SNP  ...   \n",
       "4      13225284      +      Missense_Mutation          SNP  ...   \n",
       "\n",
       "             FILTER       CONTEXT                            src_vcf_id  \\\n",
       "0              PASS  GGCTCCGGGCCG  b06a171d-e6f1-4f81-b9e3-e5e4522ccf5d   \n",
       "1  panel_of_normals   TTACTCGGTGA  b06a171d-e6f1-4f81-b9e3-e5e4522ccf5d   \n",
       "2              PASS   TCCAAGTGGCC  b06a171d-e6f1-4f81-b9e3-e5e4522ccf5d   \n",
       "3  panel_of_normals   CCCTCGGAGCT  b06a171d-e6f1-4f81-b9e3-e5e4522ccf5d   \n",
       "4              PASS   TCAAGGGCTGC  b06a171d-e6f1-4f81-b9e3-e5e4522ccf5d   \n",
       "\n",
       "                         tumor_bam_uuid                       normal_bam_uuid  \\\n",
       "0  003d6bd5-3ae8-4838-8cfc-0355ee16122d  b4ede00d-9733-4eb2-9335-c7bcd7976d2e   \n",
       "1  003d6bd5-3ae8-4838-8cfc-0355ee16122d  b4ede00d-9733-4eb2-9335-c7bcd7976d2e   \n",
       "2  003d6bd5-3ae8-4838-8cfc-0355ee16122d  b4ede00d-9733-4eb2-9335-c7bcd7976d2e   \n",
       "3  003d6bd5-3ae8-4838-8cfc-0355ee16122d  b4ede00d-9733-4eb2-9335-c7bcd7976d2e   \n",
       "4  003d6bd5-3ae8-4838-8cfc-0355ee16122d  b4ede00d-9733-4eb2-9335-c7bcd7976d2e   \n",
       "\n",
       "                                case_id GDC_FILTER  \\\n",
       "0  ec02b33c-8094-45e8-b679-b14dbe75d66f        NaN   \n",
       "1  ec02b33c-8094-45e8-b679-b14dbe75d66f    gdc_pon   \n",
       "2  ec02b33c-8094-45e8-b679-b14dbe75d66f        NaN   \n",
       "3  ec02b33c-8094-45e8-b679-b14dbe75d66f    gdc_pon   \n",
       "4  ec02b33c-8094-45e8-b679-b14dbe75d66f        NaN   \n",
       "\n",
       "                                COSMIC  MC3_Overlap  GDC_Validation_Status  \n",
       "0                                  NaN         True                Unknown  \n",
       "1                          COSM3492653         True                Unknown  \n",
       "2                          COSM4021197         True                Unknown  \n",
       "3  COSM4021467;COSM4021468;COSM4021469         True                Unknown  \n",
       "4                                  NaN        False                Unknown  \n",
       "\n",
       "[5 rows x 120 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_file = '../../data/pancancer/TCGA/mutation/download_new/c06465a3-50e7-46f7-b2dd-7bd654ca206b/TCGA.STAD.mutect.c06465a3-50e7-46f7-b2dd-7bd654ca206b.DR-10.0.somatic.maf'\n",
    "test = pd.read_csv(test_file, sep='\\t', comment='#', header=0)\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/ipykernel_launcher.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n",
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/ipykernel_launcher.py:10: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  # Remove the CWD from sys.path while we load stuff.\n"
     ]
    }
   ],
   "source": [
    "maf_shortened = test[['Hugo_Symbol', 'Tumor_Sample_Barcode', 'Chromosome', 'Start_Position',\n",
    "                         'End_Position', 'Variant_Classification', 'Reference_Allele',\n",
    "                         'Tumor_Seq_Allele2']]\n",
    "maf_shortened.columns = ['Gene', 'Tumor_Sample', 'Chromosome', 'Start_Position', 'End_Position',\n",
    "                         'Variant_Classification', 'Reference_Allele', 'Tumor_Allele']\n",
    "maf_shortened['Gene'] = maf_shortened.Gene.str.strip()\n",
    "maf_shortened.set_index('Gene', inplace=True)\n",
    "trim_barcode_to_patient = lambda x: '-'.join(str(x).split('-')[:3]) # TCGA barcode until patient\n",
    "maf_shortened.loc[:, 'Tumor_Sample'] = maf_shortened.Tumor_Sample.map(trim_barcode_to_patient)\n",
    "maf_shortened['Tumor_Type'] = 'Stomach Adenocarcinoma'\n",
    "#maf_shortened.to_csv('../../data/pancancer/TCGA/mutation/pancan_maf/STAD_onlyrequiredcols.maf', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TTN      835\n",
       "MUC16    314\n",
       "Name: Gene, dtype: int64"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAAR+ElEQVR4nO3df6zd9V3H8edL2A+dZi3j2jRt58WsccHEMdJAly1mQiyFmZU/5sKySLM06T9oZmKiRROJm0vYP+JIlEhGtTNThtOFBhbxWliMfwwogzGgw94hhDZAu7Xgj8Up8+0f53Prsd7LPbc999z2fp6P5OR8vu/v53zP53O5vM73fs73nKaqkCT14UdWegCSpMkx9CWpI4a+JHXE0Jekjhj6ktSRC1d6AG/k4osvrunp6ZUehiSdVx577LHvVtXUfPvO6dCfnp7m4MGDKz0MSTqvJHlhoX0u70hSRwx9SerISKGfZE2SLyf5dpJDSd6X5KIkM0kOt/u1rW+S3J5kNsmTSS4fOs7O1v9wkp3LNSlJ0vxGPdP/HPC3VfVu4D3AIWAPcKCqNgMH2jbAtcDmdtsN3AGQ5CLgFuBK4ArglrkXCknSZCwa+kneDvw8cBdAVf1nVb0K7AD2tW77gOtbewfwhRr4OrAmyXrgGmCmqk5U1UlgBtg+xrlIkhYxypn+JcBx4E+TPJ7k80neBqyrqpdan5eBda29AXhx6PFHWm2huiRpQkYJ/QuBy4E7quq9wL/zv0s5ANTgqzrH8nWdSXYnOZjk4PHjx8dxSElSM0roHwGOVNXDbfvLDF4EXmnLNrT7Y23/UWDT0OM3ttpC9f+jqu6sqi1VtWVqat7PFkiSztCioV9VLwMvJvmZVroaeAbYD8xdgbMTuLe19wM3tqt4tgKvtWWgB4BtSda2N3C3tZokaUJG/UTurwFfTPJm4DngEwxeMO5Jsgt4Afho6/tV4DpgFvh+60tVnUjyaeDR1u9TVXViLLMYwfSe+0+1n7/1Q5N6Wkk6p4wU+lX1BLBlnl1Xz9O3gJsWOM5eYO8SxidJGiM/kStJHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI6M9A+jn6+m99y/0kOQpHOKZ/qS1BFDX5I6YuhLUkcMfUnqiKEvSR0ZKfSTPJ/kW0meSHKw1S5KMpPkcLtf2+pJcnuS2SRPJrl86Dg7W//DSXYuz5QkSQtZypn+L1TVZVW1pW3vAQ5U1WbgQNsGuBbY3G67gTtg8CIB3AJcCVwB3DL3QiFJmoyzWd7ZAexr7X3A9UP1L9TA14E1SdYD1wAzVXWiqk4CM8D2s3h+SdISjRr6BfxdkseS7G61dVX1Umu/DKxr7Q3Ai0OPPdJqC9UlSRMy6idyP1BVR5P8JDCT5NvDO6uqktQ4BtReVHYDvPOd7xzHISVJzUhn+lV1tN0fA77CYE3+lbZsQ7s/1rofBTYNPXxjqy1UP/257qyqLVW1ZWpqammzkSS9oUVDP8nbkvzEXBvYBjwF7AfmrsDZCdzb2vuBG9tVPFuB19oy0APAtiRr2xu421pNkjQhoyzvrAO+kmSu/19U1d8meRS4J8ku4AXgo63/V4HrgFng+8AnAKrqRJJPA4+2fp+qqhNjm4kkaVGLhn5VPQe8Z57694Cr56kXcNMCx9oL7F36MCVJ4+AnciWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHVk5NBPckGSx5Pc17YvSfJwktkkX0ry5lZ/S9uebfunh45xc6s/m+Sasc9GkvSGlnKm/0ng0ND2Z4HbqupdwElgV6vvAk62+m2tH0kuBW4AfhbYDvxxkgvObviSpKUYKfSTbAQ+BHy+bQe4Cvhy67IPuL61d7Rt2v6rW/8dwN1V9YOq+mdgFrhiDHOQJI1o1DP9PwR+E/jvtv0O4NWqer1tHwE2tPYG4EWAtv+11v9UfZ7HSJImYNHQT/JLwLGqemwC4yHJ7iQHkxw8fvz4JJ5Skroxypn++4EPJ3keuJvBss7ngDVJLmx9NgJHW/sosAmg7X878L3h+jyPOaWq7qyqLVW1ZWpqaskTkiQtbNHQr6qbq2pjVU0zeCP2war6OPAQ8JHWbSdwb2vvb9u0/Q9WVbX6De3qnkuAzcAjY5uJJGlRFy7eZUG/Bdyd5PeBx4G7Wv0u4M+TzAInGLxQUFVPJ7kHeAZ4Hbipqn54Fs8vSVqiJYV+VX0N+FprP8c8V99U1X8Av7zA4z8DfGapg5QkjYefyJWkjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SerI2Xy18nlres/9p9rP3/qhFRyJJE2WZ/qS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdWTT0k7w1ySNJvpnk6SS/1+qXJHk4yWySLyV5c6u/pW3Ptv3TQ8e6udWfTXLNss1KkjSvUc70fwBcVVXvAS4DtifZCnwWuK2q3gWcBHa1/ruAk61+W+tHkkuBG4CfBbYDf5zkgjHORZK0iEVDvwb+rW2+qd0KuAr4cqvvA65v7R1tm7b/6iRp9bur6gdV9c/ALHDFOCYhSRrNSGv6SS5I8gRwDJgBvgO8WlWvty5HgA2tvQF4EaDtfw14x3B9nscMP9fuJAeTHDx+/PiSJyRJWthIoV9VP6yqy4CNDM7O371cA6qqO6tqS1VtmZqaWq6nkaQuLenqnap6FXgIeB+wJsncP7e4ETja2keBTQBt/9uB7w3X53mMJGkCRrl6ZyrJmtb+UeAXgUMMwv8jrdtO4N7W3t+2afsfrKpq9Rva1T2XAJuBR8Y0D0nSCEb5h9HXA/valTY/AtxTVfcleQa4O8nvA48Dd7X+dwF/nmQWOMHgih2q6ukk9wDPAK8DN1XVD8c7HUnSG1k09KvqSeC989SfY56rb6rqP4BfXuBYnwE+s/RhSpLGwU/kSlJHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6siioZ9kU5KHkjyT5Okkn2z1i5LMJDnc7te2epLcnmQ2yZNJLh861s7W/3CSncs3LUnSfEY5038d+I2quhTYCtyU5FJgD3CgqjYDB9o2wLXA5nbbDdwBgxcJ4BbgSuAK4Ja5FwpJ0mQsGvpV9VJVfaO1/xU4BGwAdgD7Wrd9wPWtvQP4Qg18HViTZD1wDTBTVSeq6iQwA2wf52QkSW9sSWv6SaaB9wIPA+uq6qW262VgXWtvAF4cetiRVluofvpz7E5yMMnB48ePL2V4kqRFjBz6SX4c+Gvg16vqX4b3VVUBNY4BVdWdVbWlqrZMTU2N45CSpGak0E/yJgaB/8Wq+ptWfqUt29Duj7X6UWDT0MM3ttpCdUnShIxy9U6Au4BDVfUHQ7v2A3NX4OwE7h2q39iu4tkKvNaWgR4AtiVZ297A3dZqK2p6z/2nbpK02l04Qp/3A78CfCvJE63228CtwD1JdgEvAB9t+74KXAfMAt8HPgFQVSeSfBp4tPX7VFWdGMckJEmjWTT0q+ofgSyw++p5+hdw0wLH2gvsXcoAJUnj4ydyJakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdWTR0E+yN8mxJE8N1S5KMpPkcLtf2+pJcnuS2SRPJrl86DE7W//DSXYuz3QkSW9klDP9PwO2n1bbAxyoqs3AgbYNcC2wud12A3fA4EUCuAW4ErgCuGXuhUKSNDmLhn5V/QNw4rTyDmBfa+8Drh+qf6EGvg6sSbIeuAaYqaoTVXUSmOH/v5BIkpbZma7pr6uql1r7ZWBda28AXhzqd6TVFqr/P0l2JzmY5ODx48fPcHiSpPmc9Ru5VVVAjWEsc8e7s6q2VNWWqampcR1WkgRceIaPeyXJ+qp6qS3fHGv1o8CmoX4bW+0o8MHT6l87w+deNtN77j/Vfv7WD63gSCRpeZzpmf5+YO4KnJ3AvUP1G9tVPFuB19oy0APAtiRr2xu421pNkjRBi57pJ/lLBmfpFyc5wuAqnFuBe5LsAl4APtq6fxW4DpgFvg98AqCqTiT5NPBo6/epqjr9zWFJ0jJbNPSr6mML7Lp6nr4F3LTAcfYCe5c0OknSWPmJXEnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdOdNv2Vz1/MZNSauRZ/qS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI16yOQIv35S0WnimL0kdMfQlqSOGviR1xDX9JXJ9X9L5zDN9SeqIoS9JHXF55yy41CPpfOOZviR1ZOJn+km2A58DLgA+X1W3TnoMy2H4rH+YfwFIOpdMNPSTXAD8EfCLwBHg0ST7q+qZSY5jklwCknQumfSZ/hXAbFU9B5DkbmAHsGpDf9hCfw2cCV9AJJ2JSYf+BuDFoe0jwJXDHZLsBna3zX9L8uxZPN/FwHfP4vHnrHx2wV2rds6L6HHezrkfS533Ty2045y7eqeq7gTuHMexkhysqi3jONb5osc5Q5/zds79GOe8J331zlFg09D2xlaTJE3ApEP/UWBzkkuSvBm4Adg/4TFIUrcmurxTVa8n+VXgAQaXbO6tqqeX8SnHskx0nulxztDnvJ1zP8Y271TVuI4lSTrH+YlcSeqIoS9JHVmVoZ9ke5Jnk8wm2bPS4xmnJHuTHEvy1FDtoiQzSQ63+7WtniS3t5/Dk0kuX7mRn7kkm5I8lOSZJE8n+WSrr9p5J3lrkkeSfLPN+fda/ZIkD7e5faldEEGSt7Tt2bZ/ekUncJaSXJDk8ST3te1VPe8kzyf5VpInkhxstWX5/V51oT/0VQ/XApcCH0ty6cqOaqz+DNh+Wm0PcKCqNgMH2jYMfgab2203cMeExjhurwO/UVWXAluBm9p/09U87x8AV1XVe4DLgO1JtgKfBW6rqncBJ4Fdrf8u4GSr39b6nc8+CRwa2u5h3r9QVZcNXY+/PL/fVbWqbsD7gAeGtm8Gbl7pcY15jtPAU0PbzwLrW3s98Gxr/wnwsfn6nc834F4G39/UxbyBHwO+weDT698FLmz1U7/rDK6Ie19rX9j6ZaXHfobz3dhC7irgPiCrfd7A88DFp9WW5fd71Z3pM/9XPWxYobFMyrqqeqm1XwbWtfaq+1m0P9/fCzzMKp93W+J4AjgGzADfAV6tqtdbl+F5nZpz2/8a8I6JDnh8/hD4TeC/2/Y7WP3zLuDvkjzWvooGlun3+5z7GgadnaqqJKvyOtwkPw78NfDrVfUvSU7tW43zrqofApclWQN8BXj3yo5o+SX5JeBYVT2W5IMrPJxJ+kBVHU3yk8BMkm8P7xzn7/dqPNPv8aseXkmyHqDdH2v1VfOzSPImBoH/xar6m1Ze9fMGqKpXgYcYLGusSTJ3sjY8r1NzbvvfDnxvsiMdi/cDH07yPHA3gyWez7HK511VR9v9MQYv8FewTL/fqzH0e/yqh/3AztbeyWDNe65+Y3u3fyvw2tCfi+eNDE7p7wIOVdUfDO1atfNOMtXO8EnyowzewzjEIPw/0rqdPue5n8VHgAerLfieT6rq5qraWFXTDP7ffbCqPs4qnneStyX5ibk2sA14iuX6/V7pNzCW6U2R64B/YrAG+jsrPZ4xz+0vgZeA/2KwlreLwRrmAeAw8PfARa1vGFzJ9B3gW8CWlR7/Gc75AwzWPJ8Enmi361bzvIGfAx5vc34K+N1W/2ngEWAW+CvgLa3+1rY92/b/9ErPYQw/gw8C9632ebe5fbPdnp7LrOX6/fZrGCSpI6txeUeStABDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXkfwDwAqYwlVrqzQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(maf_shortened.index.value_counts(), bins=np.arange(0, 500, 5))\n",
    "t = maf_shortened.index.value_counts()\n",
    "t[t >= 300]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "test2 = pd.read_csv('/project/gcn/2020plus-1.2.3/data/mutations.txt', sep='\\t')\n",
    "test2['Gene'] = test2.Gene.str.strip()\n",
    "test2.set_index('Gene', inplace=True)\n",
    "test2.drop('Protein_Change', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Missense_Mutation           465597\n",
       "Silent                      177737\n",
       "Nonsense_Mutation            34194\n",
       "Splice_Site                  24459\n",
       "Frame_Shift_Indel            18691\n",
       "In_Frame_Indel                4456\n",
       "Promoter                      3004\n",
       "Translation_Start_Site         497\n",
       "Nonstop_Mutation               276\n",
       "Splice_Region                  132\n",
       "De_novo_Start_OutOfFrame        82\n",
       "NCSD                            29\n",
       "miRNA                           22\n",
       "De_novo_Start_InFrame           16\n",
       "downstream                       5\n",
       "upstream                         3\n",
       "De_novo_Start                    2\n",
       "upstream;downstream              1\n",
       "Read-through                     1\n",
       "Start_Codon_Del                  1\n",
       "Name: Variant_Classification, dtype: int64"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test2.Variant_Classification.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tumor_Sample</th>\n",
       "      <th>Tumor_Type</th>\n",
       "      <th>Chromosome</th>\n",
       "      <th>Start_Position</th>\n",
       "      <th>End_Position</th>\n",
       "      <th>Variant_Classification</th>\n",
       "      <th>Reference_Allele</th>\n",
       "      <th>Tumor_Allele</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Gene</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>IVL</th>\n",
       "      <td>BR-0001</td>\n",
       "      <td>Breast Adenocarcinoma</td>\n",
       "      <td>chr1</td>\n",
       "      <td>152883016</td>\n",
       "      <td>152883016</td>\n",
       "      <td>Missense_Mutation</td>\n",
       "      <td>A</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ZBTB37</th>\n",
       "      <td>BR-0001</td>\n",
       "      <td>Breast Adenocarcinoma</td>\n",
       "      <td>chr1</td>\n",
       "      <td>173837392</td>\n",
       "      <td>173837392</td>\n",
       "      <td>Splice_Site</td>\n",
       "      <td>AGGCA</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CHIT1</th>\n",
       "      <td>BR-0001</td>\n",
       "      <td>Breast Adenocarcinoma</td>\n",
       "      <td>chr1</td>\n",
       "      <td>203191404</td>\n",
       "      <td>203191404</td>\n",
       "      <td>Missense_Mutation</td>\n",
       "      <td>C</td>\n",
       "      <td>T</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DRD5</th>\n",
       "      <td>BR-0001</td>\n",
       "      <td>Breast Adenocarcinoma</td>\n",
       "      <td>chr4</td>\n",
       "      <td>9784617</td>\n",
       "      <td>9784617</td>\n",
       "      <td>Missense_Mutation</td>\n",
       "      <td>T</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GRM4</th>\n",
       "      <td>BR-0001</td>\n",
       "      <td>Breast Adenocarcinoma</td>\n",
       "      <td>chr6</td>\n",
       "      <td>34100851</td>\n",
       "      <td>34100852</td>\n",
       "      <td>Frame_Shift_Indel</td>\n",
       "      <td>-</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Tumor_Sample             Tumor_Type Chromosome  Start_Position  \\\n",
       "Gene                                                                    \n",
       "IVL         BR-0001  Breast Adenocarcinoma       chr1       152883016   \n",
       "ZBTB37      BR-0001  Breast Adenocarcinoma       chr1       173837392   \n",
       "CHIT1       BR-0001  Breast Adenocarcinoma       chr1       203191404   \n",
       "DRD5        BR-0001  Breast Adenocarcinoma       chr4         9784617   \n",
       "GRM4        BR-0001  Breast Adenocarcinoma       chr6        34100851   \n",
       "\n",
       "        End_Position Variant_Classification Reference_Allele Tumor_Allele  \n",
       "Gene                                                                       \n",
       "IVL        152883016      Missense_Mutation                A            G  \n",
       "ZBTB37     173837392            Splice_Site            AGGCA            -  \n",
       "CHIT1      203191404      Missense_Mutation                C            T  \n",
       "DRD5         9784617      Missense_Mutation                T            C  \n",
       "GRM4        34100852      Frame_Shift_Indel                -            G  "
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "test2.to_csv('/project/gcn/2020plus-1.2.3/data/bladder_redone.txt', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2385673, 7)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bed = pd.read_csv('/project/gcn/2020plus-1.2.3/data/snvboxGenes.bed', sep='\\t', header=None)\n",
    "bed = bed.iloc[:, :4]\n",
    "bed.columns = ['Chr', 'Start', 'End', 'Gene']\n",
    "maf_shortened[maf_shortened.index.isin(bed.Gene)].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2469816, 7)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maf_shortened.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "summary_test = pd.read_csv('/project/gcn/2020plus-1.2.3/summary_test.txt', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['GRCh38'], dtype=object)"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maf_all['NCBI_Build'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = pd.read_csv('/project/gcn/2020plus-1.2.3/data/bladder_redone.txt', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gene</th>\n",
       "      <th>Tumor_Sample</th>\n",
       "      <th>Tumor_Type</th>\n",
       "      <th>Chromosome</th>\n",
       "      <th>Start_Position</th>\n",
       "      <th>End_Position</th>\n",
       "      <th>Variant_Classification</th>\n",
       "      <th>Reference_Allele</th>\n",
       "      <th>Tumor_Allele</th>\n",
       "      <th>Protein_Change</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>IVL</td>\n",
       "      <td>BR-0001</td>\n",
       "      <td>Breast Adenocarcinoma</td>\n",
       "      <td>chr1</td>\n",
       "      <td>152883016</td>\n",
       "      <td>152883016</td>\n",
       "      <td>Missense_Mutation</td>\n",
       "      <td>A</td>\n",
       "      <td>G</td>\n",
       "      <td>p.Q248R</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ZBTB37</td>\n",
       "      <td>BR-0001</td>\n",
       "      <td>Breast Adenocarcinoma</td>\n",
       "      <td>chr1</td>\n",
       "      <td>173837392</td>\n",
       "      <td>173837392</td>\n",
       "      <td>Splice_Site</td>\n",
       "      <td>AGGCA</td>\n",
       "      <td>-</td>\n",
       "      <td>p.?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CHIT1</td>\n",
       "      <td>BR-0001</td>\n",
       "      <td>Breast Adenocarcinoma</td>\n",
       "      <td>chr1</td>\n",
       "      <td>203191404</td>\n",
       "      <td>203191404</td>\n",
       "      <td>Missense_Mutation</td>\n",
       "      <td>C</td>\n",
       "      <td>T</td>\n",
       "      <td>p.E219K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DRD5</td>\n",
       "      <td>BR-0001</td>\n",
       "      <td>Breast Adenocarcinoma</td>\n",
       "      <td>chr4</td>\n",
       "      <td>9784617</td>\n",
       "      <td>9784617</td>\n",
       "      <td>Missense_Mutation</td>\n",
       "      <td>T</td>\n",
       "      <td>C</td>\n",
       "      <td>p.C322R</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GRM4</td>\n",
       "      <td>BR-0001</td>\n",
       "      <td>Breast Adenocarcinoma</td>\n",
       "      <td>chr6</td>\n",
       "      <td>34100851</td>\n",
       "      <td>34100852</td>\n",
       "      <td>Frame_Shift_Indel</td>\n",
       "      <td>-</td>\n",
       "      <td>G</td>\n",
       "      <td>p.K141fs*</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Gene Tumor_Sample             Tumor_Type Chromosome  Start_Position  \\\n",
       "0     IVL      BR-0001  Breast Adenocarcinoma       chr1       152883016   \n",
       "1  ZBTB37      BR-0001  Breast Adenocarcinoma       chr1       173837392   \n",
       "2   CHIT1      BR-0001  Breast Adenocarcinoma       chr1       203191404   \n",
       "3    DRD5      BR-0001  Breast Adenocarcinoma       chr4         9784617   \n",
       "4    GRM4      BR-0001  Breast Adenocarcinoma       chr6        34100851   \n",
       "\n",
       "   End_Position Variant_Classification Reference_Allele Tumor_Allele  \\\n",
       "0     152883016      Missense_Mutation                A            G   \n",
       "1     173837392            Splice_Site            AGGCA            -   \n",
       "2     203191404      Missense_Mutation                C            T   \n",
       "3       9784617      Missense_Mutation                T            C   \n",
       "4      34100852      Frame_Shift_Indel                -            G   \n",
       "\n",
       "  Protein_Change  \n",
       "0        p.Q248R  \n",
       "1            p.?  \n",
       "2        p.E219K  \n",
       "3        p.C322R  \n",
       "4      p.K141fs*  "
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "112845445"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maf_shortened[maf_shortened.index == 'APC'].Start_Position.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "112179759"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test2[test2.index == 'APC'].Start_Position.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Chr</th>\n",
       "      <th>Start</th>\n",
       "      <th>End</th>\n",
       "      <th>Gene</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>789</th>\n",
       "      <td>chr5</td>\n",
       "      <td>112073555</td>\n",
       "      <td>112181936</td>\n",
       "      <td>APC</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Chr      Start        End Gene\n",
       "789  chr5  112073555  112181936  APC"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bed[bed.Gene == 'APC']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (87,98,118) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n",
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (87,88,98,118) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n",
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (87,88,118) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n",
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (19,20,28,87,88) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n",
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (87) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n",
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (88,118) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n",
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (87,118) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n",
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (88,98) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "16"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count = 0\n",
    "maf_all = None\n",
    "col_names = None\n",
    "root_dir = '../../data/pancancer/TCGA/mutation/download_new/'\n",
    "for subdir, d, files in os.walk(root_dir):\n",
    "    for fname in files:\n",
    "        if fname.endswith('.maf'):\n",
    "            p = os.path.join(subdir, fname)\n",
    "            if maf_all is None:\n",
    "                maf_all = pd.read_csv(p, sep='\\t', comment='#', header=0)\n",
    "            else:\n",
    "                maf_new = pd.read_csv(p, sep='\\t', comment='#', header=0)\n",
    "                maf_join = pd.concat([maf_all, maf_new], ignore_index=True, join='inner')\n",
    "                maf_join.dropna(axis=1, inplace=True, how='all') # remove stupid all-na columns\n",
    "                maf_all = maf_join\n",
    "            count += 1\n",
    "count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/pkg/python-3.7.7-0/lib/python3.7/site-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "bed_maf = maf_shortened[['Chromosome', 'Start_Position', 'End_Position',\n",
    "                         'Tumor_Sample', 'Variant_Classification',\n",
    "                         'Reference_Allele', 'Tumor_Allele']]\n",
    "bed_maf['Gene'] = bed_maf.index\n",
    "bed_maf.set_index('Chromosome', inplace=True)\n",
    "bed_maf.to_csv('/project/gcn/UCSCliftover/maf_bed_pancan16.maf', sep='\\t', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2469816, 7)"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bed_maf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
